{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "A100",
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LEj76j5G4INX",
        "outputId": "77f2277b-3e0f-4e6a-ca75-7122b7c3c220"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# !git clone https://github.com/THUDM/GLM-4.git"
      ],
      "metadata": {
        "id": "4yho52TV5-eG"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4nLGQV7b-Khe",
        "outputId": "b495c117-5653-4e7f-e40f-2e3ec6fee148"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "drive  sample_data\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!ls /content/drive/MyDrive/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q_-dBXnb-Ouc",
        "outputId": "58fbaca5-5275-4379-f9cb-1864784ba201"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " AdvertiseGen_fix\t     cifar-10\t\t\t\t\t SadTalker\n",
            " alpaca-lora\t\t    'Colab Notebooks'\t\t\t\t sampledata\n",
            " chapter7\t\t     deepfm.png\t\t\t\t\t sst2\n",
            " ChatGLM3\t\t     detr_demo-da2a99e9.pth\t\t\t stanford_alpaca\n",
            " chatglm3-6b\t\t     din.png\t\t\t\t\t tensorflow1\n",
            " chatglm-6b\t\t     finetune_speaker.json\t\t\t tloen-alpaca-lora\n",
            " ChatGLM-6B\t\t     GLM-4\t\t\t\t\t transformer-de-en\n",
            " chatglm_finetuning-dev      imdb_processed.csv\t\t\t\t videos\n",
            " ChatGLM-Finetuning-master   models--bert-base-uncased\t\t\t voice_data\n",
            " checkpoints\t\t     models--deepseek-ai--deepseek-llm-7b-chat\t widedeep.png\n",
            " checkpoint_save-1\t     NeuralCF.png\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 这里的微调数据集格式与glm3的不一样，请使用给大家的数据集\n",
        "# %cd GLM-4\n",
        "# !cp -r /content/drive/MyDrive/AdvertiseGen_fix/ finetune_demo/\n",
        "# %cd ..\n",
        "# !mv GLM-4/ /content/drive/MyDrive/"
      ],
      "metadata": {
        "id": "vLsbdlwj9ljc"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/drive/MyDrive/GLM-4/finetune_demo/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hssdYc0n-5kW",
        "outputId": "c8f65656-765b-4d1d-9c28-20415f9e0adf"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/GLM-4/finetune_demo\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 这一条需要修改peft==0.12.0\n",
        "!pip install -r requirements.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_u62q7Xc8eun",
        "outputId": "616b181b-2aa8-40fe-bf41-ababa79fb63d"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: jieba in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 1)) (0.42.1)\n",
            "Collecting datasets (from -r requirements.txt (line 2))\n",
            "  Downloading datasets-3.2.0-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: peft in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 3)) (0.14.0)\n",
            "Collecting deepspeed (from -r requirements.txt (line 4))\n",
            "  Downloading deepspeed-0.16.3.tar.gz (1.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m61.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 5)) (3.9.1)\n",
            "Collecting rouge_chinese (from -r requirements.txt (line 6))\n",
            "  Downloading rouge_chinese-1.0.3-py3-none-any.whl.metadata (7.6 kB)\n",
            "Collecting ruamel.yaml (from -r requirements.txt (line 7))\n",
            "  Downloading ruamel.yaml-0.18.10-py3-none-any.whl.metadata (23 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (3.17.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (1.26.4)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (17.0.0)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets->-r requirements.txt (line 2))\n",
            "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (2.2.2)\n",
            "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (2.32.3)\n",
            "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (4.67.1)\n",
            "Collecting xxhash (from datasets->-r requirements.txt (line 2))\n",
            "  Downloading xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting multiprocess<0.70.17 (from datasets->-r requirements.txt (line 2))\n",
            "  Downloading multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)\n",
            "Collecting fsspec<=2024.9.0,>=2023.1.0 (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets->-r requirements.txt (line 2))\n",
            "  Downloading fsspec-2024.9.0-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (3.11.11)\n",
            "Requirement already satisfied: huggingface-hub>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (0.28.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (24.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 2)) (6.0.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.11/dist-packages (from peft->-r requirements.txt (line 3)) (5.9.5)\n",
            "Requirement already satisfied: torch>=1.13.0 in /usr/local/lib/python3.11/dist-packages (from peft->-r requirements.txt (line 3)) (2.5.1+cu124)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.11/dist-packages (from peft->-r requirements.txt (line 3)) (4.48.2)\n",
            "Requirement already satisfied: accelerate>=0.21.0 in /usr/local/lib/python3.11/dist-packages (from peft->-r requirements.txt (line 3)) (1.3.0)\n",
            "Requirement already satisfied: safetensors in /usr/local/lib/python3.11/dist-packages (from peft->-r requirements.txt (line 3)) (0.5.2)\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.11/dist-packages (from deepspeed->-r requirements.txt (line 4)) (0.8.0)\n",
            "Collecting hjson (from deepspeed->-r requirements.txt (line 4))\n",
            "  Downloading hjson-3.1.0-py3-none-any.whl.metadata (2.6 kB)\n",
            "Requirement already satisfied: msgpack in /usr/local/lib/python3.11/dist-packages (from deepspeed->-r requirements.txt (line 4)) (1.1.0)\n",
            "Collecting ninja (from deepspeed->-r requirements.txt (line 4))\n",
            "  Downloading ninja-1.11.1.3-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl.metadata (5.3 kB)\n",
            "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.11/dist-packages (from deepspeed->-r requirements.txt (line 4)) (9.0.0)\n",
            "Requirement already satisfied: pydantic>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from deepspeed->-r requirements.txt (line 4)) (2.10.6)\n",
            "Collecting nvidia-ml-py (from deepspeed->-r requirements.txt (line 4))\n",
            "  Downloading nvidia_ml_py-12.570.86-py3-none-any.whl.metadata (8.7 kB)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.11/dist-packages (from nltk->-r requirements.txt (line 5)) (8.1.8)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (from nltk->-r requirements.txt (line 5)) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.11/dist-packages (from nltk->-r requirements.txt (line 5)) (2024.11.6)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.11/dist-packages (from rouge_chinese->-r requirements.txt (line 6)) (1.17.0)\n",
            "Collecting ruamel.yaml.clib>=0.2.7 (from ruamel.yaml->-r requirements.txt (line 7))\n",
            "  Downloading ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.7 kB)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 2)) (2.4.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 2)) (1.3.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 2)) (25.1.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 2)) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 2)) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 2)) (0.2.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 2)) (1.18.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub>=0.23.0->datasets->-r requirements.txt (line 2)) (4.12.2)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.11/dist-packages (from pydantic>=2.0.0->deepspeed->-r requirements.txt (line 4)) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.27.2 in /usr/local/lib/python3.11/dist-packages (from pydantic>=2.0.0->deepspeed->-r requirements.txt (line 4)) (2.27.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 2)) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 2)) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 2)) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 2)) (2025.1.31)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch>=1.13.0->peft->-r requirements.txt (line 3)) (3.4.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch>=1.13.0->peft->-r requirements.txt (line 3)) (3.1.5)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.4.127 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.4.127 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.4.127 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cudnn-cu12==9.1.0.70 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.4.5.8 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cufft-cu12==11.2.1.3 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-curand-cu12==10.3.5.147 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cusolver-cu12==11.6.1.9 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cusparse-cu12==12.3.1.170 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /usr/local/lib/python3.11/dist-packages (from torch>=1.13.0->peft->-r requirements.txt (line 3)) (2.21.5)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.13.0->peft->-r requirements.txt (line 3)) (12.4.127)\n",
            "Collecting nvidia-nvjitlink-cu12==12.4.127 (from torch>=1.13.0->peft->-r requirements.txt (line 3))\n",
            "  Downloading nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: triton==3.1.0 in /usr/local/lib/python3.11/dist-packages (from torch>=1.13.0->peft->-r requirements.txt (line 3)) (3.1.0)\n",
            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.11/dist-packages (from torch>=1.13.0->peft->-r requirements.txt (line 3)) (1.13.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from sympy==1.13.1->torch>=1.13.0->peft->-r requirements.txt (line 3)) (1.3.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets->-r requirements.txt (line 2)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets->-r requirements.txt (line 2)) (2025.1)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets->-r requirements.txt (line 2)) (2025.1)\n",
            "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.11/dist-packages (from transformers->peft->-r requirements.txt (line 3)) (0.21.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch>=1.13.0->peft->-r requirements.txt (line 3)) (3.0.2)\n",
            "Downloading datasets-3.2.0-py3-none-any.whl (480 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.6/480.6 kB\u001b[0m \u001b[31m41.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading rouge_chinese-1.0.3-py3-none-any.whl (21 kB)\n",
            "Downloading ruamel.yaml-0.18.10-py3-none-any.whl (117 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.7/117.7 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fsspec-2024.9.0-py3-none-any.whl (179 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.3/179.3 kB\u001b[0m \u001b[31m19.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading multiprocess-0.70.16-py311-none-any.whl (143 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.5/143.5 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (739 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m739.1/739.1 kB\u001b[0m \u001b[31m51.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl (363.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m363.4/363.4 MB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (13.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.8/13.8 MB\u001b[0m \u001b[31m107.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (24.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.6/24.6 MB\u001b[0m \u001b[31m83.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (883 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m883.7/883.7 kB\u001b[0m \u001b[31m58.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl (664.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m664.8/664.8 MB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl (211.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.5/211.5 MB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl (56.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.3/56.3 MB\u001b[0m \u001b[31m41.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl (127.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m127.9/127.9 MB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl (207.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.5/207.5 MB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m95.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading hjson-3.1.0-py3-none-any.whl (54 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.0/54.0 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading ninja-1.11.1.3-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (422 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m422.9/422.9 kB\u001b[0m \u001b[31m33.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_ml_py-12.570.86-py3-none-any.whl (44 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.4/44.4 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.8/194.8 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: deepspeed\n",
            "  Building wheel for deepspeed (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for deepspeed: filename=deepspeed-0.16.3-py3-none-any.whl size=1550060 sha256=f3543e6a343601a82f5a2aac5e6b307b3061a4d02d4bdb257ee750a91ee5ea41\n",
            "  Stored in directory: /root/.cache/pip/wheels/ab/dc/d4/7e7e07b11bc7c0e2a1a495b967acf58de61261eed4596fb23b\n",
            "Successfully built deepspeed\n",
            "Installing collected packages: nvidia-ml-py, hjson, xxhash, ruamel.yaml.clib, rouge_chinese, nvidia-nvjitlink-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, ninja, fsspec, dill, ruamel.yaml, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, nvidia-cusolver-cu12, datasets, deepspeed\n",
            "  Attempting uninstall: nvidia-nvjitlink-cu12\n",
            "    Found existing installation: nvidia-nvjitlink-cu12 12.5.82\n",
            "    Uninstalling nvidia-nvjitlink-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-nvjitlink-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-curand-cu12\n",
            "    Found existing installation: nvidia-curand-cu12 10.3.6.82\n",
            "    Uninstalling nvidia-curand-cu12-10.3.6.82:\n",
            "      Successfully uninstalled nvidia-curand-cu12-10.3.6.82\n",
            "  Attempting uninstall: nvidia-cufft-cu12\n",
            "    Found existing installation: nvidia-cufft-cu12 11.2.3.61\n",
            "    Uninstalling nvidia-cufft-cu12-11.2.3.61:\n",
            "      Successfully uninstalled nvidia-cufft-cu12-11.2.3.61\n",
            "  Attempting uninstall: nvidia-cuda-runtime-cu12\n",
            "    Found existing installation: nvidia-cuda-runtime-cu12 12.5.82\n",
            "    Uninstalling nvidia-cuda-runtime-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-cuda-runtime-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-cuda-nvrtc-cu12\n",
            "    Found existing installation: nvidia-cuda-nvrtc-cu12 12.5.82\n",
            "    Uninstalling nvidia-cuda-nvrtc-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-cuda-nvrtc-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-cuda-cupti-cu12\n",
            "    Found existing installation: nvidia-cuda-cupti-cu12 12.5.82\n",
            "    Uninstalling nvidia-cuda-cupti-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-cuda-cupti-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-cublas-cu12\n",
            "    Found existing installation: nvidia-cublas-cu12 12.5.3.2\n",
            "    Uninstalling nvidia-cublas-cu12-12.5.3.2:\n",
            "      Successfully uninstalled nvidia-cublas-cu12-12.5.3.2\n",
            "  Attempting uninstall: fsspec\n",
            "    Found existing installation: fsspec 2024.10.0\n",
            "    Uninstalling fsspec-2024.10.0:\n",
            "      Successfully uninstalled fsspec-2024.10.0\n",
            "  Attempting uninstall: nvidia-cusparse-cu12\n",
            "    Found existing installation: nvidia-cusparse-cu12 12.5.1.3\n",
            "    Uninstalling nvidia-cusparse-cu12-12.5.1.3:\n",
            "      Successfully uninstalled nvidia-cusparse-cu12-12.5.1.3\n",
            "  Attempting uninstall: nvidia-cudnn-cu12\n",
            "    Found existing installation: nvidia-cudnn-cu12 9.3.0.75\n",
            "    Uninstalling nvidia-cudnn-cu12-9.3.0.75:\n",
            "      Successfully uninstalled nvidia-cudnn-cu12-9.3.0.75\n",
            "  Attempting uninstall: nvidia-cusolver-cu12\n",
            "    Found existing installation: nvidia-cusolver-cu12 11.6.3.83\n",
            "    Uninstalling nvidia-cusolver-cu12-11.6.3.83:\n",
            "      Successfully uninstalled nvidia-cusolver-cu12-11.6.3.83\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "gcsfs 2024.10.0 requires fsspec==2024.10.0, but you have fsspec 2024.9.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed datasets-3.2.0 deepspeed-0.16.3 dill-0.3.8 fsspec-2024.9.0 hjson-3.1.0 multiprocess-0.70.16 ninja-1.11.1.3 nvidia-cublas-cu12-12.4.5.8 nvidia-cuda-cupti-cu12-12.4.127 nvidia-cuda-nvrtc-cu12-12.4.127 nvidia-cuda-runtime-cu12-12.4.127 nvidia-cudnn-cu12-9.1.0.70 nvidia-cufft-cu12-11.2.1.3 nvidia-curand-cu12-10.3.5.147 nvidia-cusolver-cu12-11.6.1.9 nvidia-cusparse-cu12-12.3.1.170 nvidia-ml-py-12.570.86 nvidia-nvjitlink-cu12-12.4.127 rouge_chinese-1.0.3 ruamel.yaml-0.18.10 ruamel.yaml.clib-0.2.12 xxhash-3.5.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip list|grep dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GLg48G9X7LIj",
        "outputId": "276ad4b0-f6ed-4d26-8310-881483061ded"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "datasets                           3.2.0\n",
            "tensorflow-datasets                4.9.7\n",
            "vega-datasets                      0.9.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tiktoken"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B1Iyg8tECJrd",
        "outputId": "a2f3040a-5023-4382-d2f6-d75021e1583e"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.11/dist-packages (from tiktoken) (2024.11.6)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.11/dist-packages (from tiktoken) (2.32.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.26.0->tiktoken) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.26.0->tiktoken) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests>=2.26.0->tiktoken) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.26.0->tiktoken) (2025.1.31)\n",
            "Downloading tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[?25l   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/1.2 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m51.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.8.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# !pip install transformers==4.48.1"
      ],
      "metadata": {
        "id": "JBsG3ZTEvP6p"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python finetune.py  ./AdvertiseGen_fix/  THUDM/glm-4-9b-chat  configs/lora.yaml\n",
        "# !python finetune.py  ./AdvertiseGen_fix/  THUDM/glm-4-9b-chat  configs/lora.yaml # For Chat Fine-tune\n",
        "# python finetune_vision.py  data/CogVLM-311K/  THUDM/glm-4v-9b configs/lora.yaml # For VQA Fine-tune"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GJco__BK8hMc",
        "outputId": "2b207c9d-5316-47ff-e4f7-00142792ab49"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2025-02-08 09:43:09.669182: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
            "2025-02-08 09:43:09.688009: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
            "E0000 00:00:1739007789.710454    9864 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "E0000 00:00:1739007789.717169    9864 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2025-02-08 09:43:09.740027: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "Loading checkpoint shards: 100% 10/10 [00:03<00:00,  3.10it/s]\n",
            "trainable params: 2,785,280 || all params: 9,402,736,640 || trainable%: 0.0296\n",
            "train_dataset: Dataset({\n",
            "    features: ['input_ids', 'labels'],\n",
            "    num_rows: 114599\n",
            "})\n",
            "val_dataset: Dataset({\n",
            "    features: ['input_ids', 'output_ids'],\n",
            "    num_rows: 1070\n",
            "})\n",
            "test_dataset: Dataset({\n",
            "    features: ['input_ids', 'output_ids'],\n",
            "    num_rows: 1070\n",
            "})\n",
            "[2025-02-08 09:43:27,912] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n",
            "max_steps is given, it will override any value given in num_train_epochs\n",
            "/usr/local/lib/python3.11/dist-packages/torch/utils/data/dataloader.py:617: UserWarning: This DataLoader will create 16 worker processes in total. Our suggested max number of worker in current system is 12, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(\n",
            "***** Running training *****\n",
            "  Num examples = 114,599\n",
            "  Num Epochs = 1\n",
            "  Instantaneous batch size per device = 1\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 1\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3,000\n",
            "  Number of trainable parameters = 2,785,280\n",
            "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mlizhilong1987\u001b[0m to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.19.6\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/GLM-4/finetune_demo/wandb/run-20250208_094330-ksxxkxv2\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33m./output\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/lizhilong1987/huggingface\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/lizhilong1987/huggingface/runs/ksxxkxv2\u001b[0m\n",
            "  0% 0/3000 [00:00<?, ?it/s]/usr/local/lib/python3.11/dist-packages/torch/utils/data/dataloader.py:617: UserWarning: This DataLoader will create 16 worker processes in total. Our suggested max number of worker in current system is 12, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/GLM-4/finetune_demo/finetune.py\", line 508, in <module>\n",
            "    app()\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/typer/main.py\", line 340, in __call__\n",
            "    raise e\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/typer/main.py\", line 323, in __call__\n",
            "    return get_command(self)(*args, **kwargs)\n",
            "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/click/core.py\", line 1161, in __call__\n",
            "    return self.main(*args, **kwargs)\n",
            "           ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/typer/core.py\", line 680, in main\n",
            "    return _main(\n",
            "           ^^^^^^\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/typer/core.py\", line 198, in _main\n",
            "    rv = self.invoke(ctx)\n",
            "         ^^^^^^^^^^^^^^^^\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/click/core.py\", line 1443, in invoke\n",
            "    return ctx.invoke(self.callback, **ctx.params)\n",
            "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/click/core.py\", line 788, in invoke\n",
            "    return __callback(*args, **kwargs)\n",
            "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/typer/main.py\", line 698, in wrapper\n",
            "    return callback(**use_params)\n",
            "           ^^^^^^^^^^^^^^^^^^^^^^\n",
            "  File \"/content/drive/MyDrive/GLM-4/finetune_demo/finetune.py\", line 471, in main\n",
            "    trainer.train()\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/transformers/trainer.py\", line 2171, in train\n",
            "    return inner_training_loop(\n",
            "           ^^^^^^^^^^^^^^^^^^^^\n",
            "  File \"/usr/local/lib/python3.11/dist-packages/transformers/trainer.py\", line 2531, in _inner_training_loop\n",
            "    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)\n",
            "                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
            "TypeError: Seq2SeqTrainer.training_step() takes 3 positional arguments but 4 were given\n",
            "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
            "\u001b[31m│\u001b[0m \u001b[2;33m/content/drive/MyDrive/GLM-4/finetune_demo/\u001b[0m\u001b[1;33mfinetune.py\u001b[0m:\u001b[94m471\u001b[0m in \u001b[92mmain\u001b[0m                               \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m468 \u001b[0m\u001b[2m│   \u001b[0m)                                                                                      \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m469 \u001b[0m\u001b[2m│   \u001b[0m                                                                                       \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m470 \u001b[0m\u001b[2m│   \u001b[0m\u001b[94mif\u001b[0m auto_resume_from_checkpoint.upper() == \u001b[33m\"\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mor\u001b[0m auto_resume_from_checkpoint \u001b[95mis\u001b[0m \u001b[94mNone\u001b[0m:   \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m471 \u001b[2m│   │   \u001b[0m\u001b[1;4mtrainer.train()\u001b[0m                                                                    \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m472 \u001b[0m\u001b[2m│   \u001b[0m\u001b[94melse\u001b[0m:                                                                                  \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m473 \u001b[0m\u001b[2m│   │   \u001b[0moutput_dir = ft_config.training_args.output_dir                                    \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m474 \u001b[0m\u001b[2m│   │   \u001b[0mdirlist = os.listdir(output_dir)                                                   \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m \u001b[2;33m/usr/local/lib/python3.11/dist-packages/transformers/\u001b[0m\u001b[1;33mtrainer.py\u001b[0m:\u001b[94m2171\u001b[0m in \u001b[92mtrain\u001b[0m                    \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2168 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mfinally\u001b[0m:                                                                      \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2169 \u001b[0m\u001b[2m│   │   │   │   \u001b[0mhf_hub_utils.enable_progress_bars()                                       \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2170 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94melse\u001b[0m:                                                                             \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2171 \u001b[2m│   │   │   \u001b[0m\u001b[94mreturn\u001b[0m inner_training_loop(                                                   \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2172 \u001b[0m\u001b[2m│   │   │   │   \u001b[0margs=args,                                                                \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2173 \u001b[0m\u001b[2m│   │   │   │   \u001b[0mresume_from_checkpoint=resume_from_checkpoint,                            \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2174 \u001b[0m\u001b[2m│   │   │   │   \u001b[0mtrial=trial,                                                              \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m \u001b[2;33m/usr/local/lib/python3.11/dist-packages/transformers/\u001b[0m\u001b[1;33mtrainer.py\u001b[0m:\u001b[94m2531\u001b[0m in \u001b[92m_inner_training_loop\u001b[0m     \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2528 \u001b[0m\u001b[2m│   │   │   │   │   │   \u001b[0m\u001b[94melse\u001b[0m contextlib.nullcontext                                       \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2529 \u001b[0m\u001b[2m│   │   │   │   │   \u001b[0m)                                                                     \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2530 \u001b[0m\u001b[2m│   │   │   │   │   \u001b[0m\u001b[94mwith\u001b[0m context():                                                       \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2531 \u001b[2m│   │   │   │   │   │   \u001b[0mtr_loss_step = \u001b[1;4;96mself\u001b[0m\u001b[1;4m.training_step(model, inputs, num_items_in_ba\u001b[0m  \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2532 \u001b[0m\u001b[2m│   │   │   │   │   \u001b[0m                                                                      \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2533 \u001b[0m\u001b[2m│   │   │   │   │   \u001b[0m\u001b[94mif\u001b[0m (                                                                  \u001b[31m│\u001b[0m\n",
            "\u001b[31m│\u001b[0m   \u001b[2m2534 \u001b[0m\u001b[2m│   │   │   │   │   │   \u001b[0margs.logging_nan_inf_filter                                       \u001b[31m│\u001b[0m\n",
            "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
            "\u001b[1;91mTypeError: \u001b[0m\u001b[1;35mSeq2SeqTrainer.training_step\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m takes \u001b[1;36m3\u001b[0m positional arguments but \u001b[1;36m4\u001b[0m were given\n",
            "\u001b[1;34mwandb\u001b[0m: \n",
            "\u001b[1;34mwandb\u001b[0m: 🚀 View run \u001b[33m./output\u001b[0m at: \u001b[34mhttps://wandb.ai/lizhilong1987/huggingface/runs/ksxxkxv2\u001b[0m\n",
            "\u001b[1;34mwandb\u001b[0m: Find logs at: \u001b[1;35mwandb/run-20250208_094330-ksxxkxv2/logs\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.listdir('output/checkpoint-3000')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DxYHCxfaYw7R",
        "outputId": "a3d41c69-4a68-4a7f-8910-5d00703753f4"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['README.md',\n",
              " 'adapter_model.safetensors',\n",
              " 'adapter_config.json',\n",
              " 'training_args.bin',\n",
              " 'optimizer.pt',\n",
              " 'scheduler.pt',\n",
              " 'rng_state.pth',\n",
              " 'trainer_state.json']"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#默认使用的原始大模型路径就是在系统下，这里跟的是微调后的权重\n",
        "#如果要预测其他内容，直接修改inference.py中的messages即可，也可以修改代码，传参argv\n",
        "!python inference.py output/checkpoint-3000"
      ],
      "metadata": {
        "id": "nSiWor7u9LaX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "637db675-232a-49a1-ffbb-4e1d951dd00c"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2025-02-08 09:30:06.996376: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
            "2025-02-08 09:30:07.014923: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
            "E0000 00:00:1739007007.037747    6174 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "E0000 00:00:1739007007.044685    6174 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2025-02-08 09:30:07.067361: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "Loading checkpoint shards: 100% 10/10 [00:09<00:00,  1.03it/s]\n",
            "PeftModelForCausalLM(\n",
            "  (base_model): LoraModel(\n",
            "    (model): ChatGLMForConditionalGeneration(\n",
            "      (transformer): ChatGLMModel(\n",
            "        (embedding): Embedding(\n",
            "          (word_embeddings): Embedding(151552, 4096)\n",
            "        )\n",
            "        (rotary_pos_emb): RotaryEmbedding()\n",
            "        (encoder): GLMTransformer(\n",
            "          (layers): ModuleList(\n",
            "            (0-39): 40 x GLMBlock(\n",
            "              (input_layernorm): RMSNorm()\n",
            "              (self_attention): SelfAttention(\n",
            "                (query_key_value): lora.Linear(\n",
            "                  (base_layer): Linear(in_features=4096, out_features=4608, bias=True)\n",
            "                  (lora_dropout): ModuleDict(\n",
            "                    (default): Dropout(p=0.1, inplace=False)\n",
            "                  )\n",
            "                  (lora_A): ModuleDict(\n",
            "                    (default): Linear(in_features=4096, out_features=8, bias=False)\n",
            "                  )\n",
            "                  (lora_B): ModuleDict(\n",
            "                    (default): Linear(in_features=8, out_features=4608, bias=False)\n",
            "                  )\n",
            "                  (lora_embedding_A): ParameterDict()\n",
            "                  (lora_embedding_B): ParameterDict()\n",
            "                  (lora_magnitude_vector): ModuleDict()\n",
            "                )\n",
            "                (core_attention): SdpaAttention(\n",
            "                  (attention_dropout): Dropout(p=0.0, inplace=False)\n",
            "                )\n",
            "                (dense): Linear(in_features=4096, out_features=4096, bias=False)\n",
            "              )\n",
            "              (post_attention_layernorm): RMSNorm()\n",
            "              (mlp): MLP(\n",
            "                (dense_h_to_4h): Linear(in_features=4096, out_features=27392, bias=False)\n",
            "                (dense_4h_to_h): Linear(in_features=13696, out_features=4096, bias=False)\n",
            "              )\n",
            "            )\n",
            "          )\n",
            "          (final_layernorm): RMSNorm()\n",
            "        )\n",
            "        (output_layer): Linear(in_features=4096, out_features=151552, bias=False)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            ")\n",
            "=========\n",
            "这是一款充满活力的时尚小裙，采用了亮丽的色彩，让你在炎热的夏季穿上它，仿佛置身于花海之中。简约的版型设计，适合各种身材穿着，展现出你优雅迷人的身姿；腰部采用收腰的设计，修饰了你的身体曲线，展现女性性感的一面。\n"
          ]
        }
      ]
    }
  ]
}